Buildings classification based on facades images

In [11]:
%load_ext autoreload
%autoreload 2

import sys
sys.path.append('../')

# sys.path.append('/../..')
# sys.path.append('/../')

from keras.backend import clear_session
import matplotlib.pyplot as plt
import numpy as np

The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
In [13]:
from building_classification_package.config import CNN_MODEL_CONFIG, \
                                                   VGG_MODEL_CONFIG
from building_classification_package.model_utils import train_model, \
                                                        load_trained_and_compiled_model, \
                                                        evaluate_model, \
                                                        model_predict

Import data

In [3]:
#this is needed in dap to unzip the files which are uploaded (unless I moved them already)

# !ls /opt/app-root/s3_home/uploads
# !unzip /opt/app-root/s3_home/uploads/buildings_data_smaller.zip -d data
In [14]:
def get_data_dir(which_set):
    return f"../data/Building_labeled_{which_set}_data"

Build training, val, test sets

In [109]:
from building_classification_package.data_utils import build_dataset
from building_classification_package.config import DATA_CONFIG

#here I import datasets preprocessed with the function keras uses to preprocess for vgg model

train_iterator = build_dataset(set_to_build='train', 
                              dataset_path=get_data_dir('train'),
                            validation_split=0.2,
                              data_config=DATA_CONFIG)

val_iterator = build_dataset(set_to_build='val',
                                dataset_path = get_data_dir('train'),
                                validation_split=0.2,
                                data_config=DATA_CONFIG)

test_iterator = build_dataset(set_to_build='test',
                                dataset_path = get_data_dir('test'),
                                data_config=DATA_CONFIG)
    

Found 8800 images belonging to 5 classes.
Found 2200 images belonging to 5 classes.
Found 1358 images belonging to 5 classes.
In [110]:
#not preprocessed sets:

DATA_CONFIG_NP = DATA_CONFIG.copy()
DATA_CONFIG_NP['preprocessing_fuction'] = None


train_iterator_nop = build_dataset(set_to_build='train', 
                              dataset_path=get_data_dir('train'),
                            validation_split=0.2,
                              data_config=DATA_CONFIG_NP)

val_iterator_nop = build_dataset(set_to_build='val',
                                dataset_path = get_data_dir('train'),
                                validation_split=0.2,
                                data_config=DATA_CONFIG_NP)

test_iterator_nop = build_dataset(set_to_build='test',
                                dataset_path = get_data_dir('test'),
                                data_config=DATA_CONFIG_NP)

Found 8800 images belonging to 5 classes.
Found 2200 images belonging to 5 classes.
Found 1358 images belonging to 5 classes.
In [111]:
#sets with data augmentation:

train_iterator_aug = build_dataset(set_to_build='train', 
                              dataset_path=get_data_dir('train'),
                            validation_split=0.2,
                              data_config=DATA_CONFIG, 
                            shear_range=0.2,zoom_range=0.2,
                            rotation_range=70,horizontal_flip=True)

val_iterator_aug = build_dataset(set_to_build='val',
                                dataset_path = get_data_dir('train'),
                                validation_split=0.2,
                                data_config=DATA_CONFIG, 
                                 shear_range=0.2,zoom_range=0.2,
                            rotation_range=70,horizontal_flip=True)

test_iterator_aug = build_dataset(set_to_build='test',
                                dataset_path = get_data_dir('test'),
                                data_config=DATA_CONFIG, 
                                  shear_range=0.2,zoom_range=0.2,
                            rotation_range=70,horizontal_flip=True)

Found 8800 images belonging to 5 classes.
Found 2200 images belonging to 5 classes.
Found 1358 images belonging to 5 classes.
In [112]:
#check sizes
x_example, y_example = train_iterator[1]

input_shape = x_example[0].shape #the input shape should not include batch size
n_classes = y_example[0].shape[0]

print('size of each image: ', input_shape)
print('number of classes: ', n_classes)

size of each image:  (256, 256, 3)
number of classes:  5

Test that the sets work

In [113]:
X_train, y_train = train_iterator[1]
X_val, y_val = val_iterator[1]
X_test, y_test = test_iterator[1]

print ('X_train: ', X_train.shape,
       '\ny_train: ', y_train.shape,
       '\nX_val: ', X_val.shape,
       '\ny_val: ', y_val.shape,
       '\nX_test: ', X_test.shape,
       '\ny_test: ', y_test.shape)

X_train:  (32, 256, 256, 3) 
y_train:  (32, 5) 
X_val:  (32, 256, 256, 3) 
y_val:  (32, 5) 
X_test:  (32, 256, 256, 3) 
y_test:  (32, 5)
In [117]:
#print an example (both preprocessed and not preprocessed)
x,y = train_iterator[0]
x_nop, y_nop = train_iterator_nop[0]
for i in range(0,1):
    image = x[i] 
    image_nop = x_nop[i]
    plt.imshow(image_nop)
    plt.show()
    print('processed:')
    plt.imshow(image)
    plt.show()

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

processed:
In [21]:
#print an example (with and without augmentation)
x_aug, y_aug = train_iterator_aug[0]
for i in range(0,1):
    image = x[i] 
    image_aug = x_aug[i]
    plt.imshow(image)
    plt.show()
    print('augmented:')
    plt.imshow(image_aug)
    plt.show()

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

augmented:

CNN model

The first model I try is a simple CNN model with 2 convolution layers

In [22]:
clear_session()
In [23]:
train_model(train_data=train_iterator,
                  val_data=val_iterator,
                 model_config=CNN_MODEL_CONFIG)

275/275 [==============================] - 63s 230ms/step - loss: 1.1896 - accuracy: 0.5100 - val_loss: 1.2346 - val_accuracy: 0.5023
Epoch 00048: early stopping
Out[23]:
<building_classification_package.config.SimpleCnnModel at 0x7f7b9bc865f8>
In [43]:
#loss and accuracy:
print('validation set:')
print(evaluate_model(val_data = val_iterator, model_config=CNN_MODEL_CONFIG))

#accuracy on test
print ('test set:')
evaluate_model(val_data = test_iterator, model_config=CNN_MODEL_CONFIG)

validation set:
69/69 [==============================] - 13s 182ms/step - loss: 1.2359 - accuracy: 0.5023
test set:
43/43 [==============================] - 12s 288ms/step - loss: 1.3929 - accuracy: 0.4065
Out[43]:
[1.3929476765699165, 0.4064801]
In [83]:
cnn_predictions = model_predict(CNN_MODEL_CONFIG, val_iterator)
In [26]:
cnn_predictions
Out[26]:
array([[0.14054592, 0.12043808, 0.26235   , 0.3042411 , 0.17242496],
       [0.2749589 , 0.1714861 , 0.15366647, 0.20895046, 0.19093809],
       [0.23325019, 0.19765735, 0.14166617, 0.16847545, 0.2589508 ],
       ...,
       [0.07402896, 0.08645745, 0.5082502 , 0.2847149 , 0.04654849],
       [0.23277266, 0.04754169, 0.10082592, 0.40290448, 0.21595521],
       [0.17644213, 0.3836558 , 0.23392826, 0.12192918, 0.08404466]],
      dtype=float32)

Transfer learning with VGG16

The second model I try is based on pre-trained VGG16 as base model.

In [36]:
clear_session()

/usr/local/second-app-dir/venvpy3/lib/python3.6/site-packages/keras/engine/saving.py:341: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.
  warnings.warn('No training configuration found in save file: '
In [41]:
from building_classification_package.config import VGG_MODEL_CONFIG

/usr/local/second-app-dir/venvpy3/lib/python3.6/site-packages/keras/engine/saving.py:341: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.
  warnings.warn('No training configuration found in save file: '
In [42]:
train_model(train_iterator, val_iterator, VGG_MODEL_CONFIG)

Epoch 00047: early stopping
Out[42]:
<keras.engine.training.Model at 0x7f799c5afba8>
In [46]:
#loss and accuracy
print('validation set:')
print(evaluate_model(val_data = val_iterator, model_config=VGG_MODEL_CONFIG))

#on unseen test set
print('test set:')
evaluate_model(val_data = test_iterator, model_config=VGG_MODEL_CONFIG)

validation set:
69/69 [==============================] - 12s 170ms/step
[0.9671871662139893, 0.6036363840103149]
test set:
43/43 [==============================] - 8s 179ms/step
Out[46]:
[1.2672785520553589, 0.5147275328636169]
In [84]:
vgg_predictions = model_predict(VGG_MODEL_CONFIG, val_iterator)

/usr/local/second-app-dir/venvpy3/lib/python3.6/site-packages/keras/engine/saving.py:341: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.
  warnings.warn('No training configuration found in save file: '
In [48]:
vgg_predictions
Out[48]:
array([[0.06100238, 0.02117618, 0.22034258, 0.4940376 , 0.20344128],
       [0.18680131, 0.08674432, 0.27442047, 0.2682145 , 0.1838194 ],
       [0.00906832, 0.00179091, 0.13843784, 0.71406275, 0.13664019],
       ...,
       [0.07716348, 0.05379067, 0.5178184 , 0.26013616, 0.09109136],
       [0.12703222, 0.1042867 , 0.22864196, 0.36816663, 0.17187256],
       [0.21709998, 0.3398552 , 0.10439418, 0.19641265, 0.14223798]],
      dtype=float32)

Check mislabeled examples

In [189]:
labels = val_iterator.class_indices
labels_flip = {value:key for key, value in labels.items()}

labels_flip
Out[189]:
{0: 'apartment', 1: 'house', 2: 'industrial', 3: 'retail', 4: 'officebuilding'}
In [190]:
#real y values of validation set
real_y = val_iterator.labels

#extract some images and their labels
val_iterator.reset()
val_sample = [next(val_iterator_nop) for _ in range(150)]
x_val_no_p = [x[0] for x in val_sample]

#prediction on validation set
pred_proba = vgg_predictions
pred = [np.argmax(x) for x in pred_proba]
In [191]:
#some examples of correctly labeled images

for p, image, y, probs in zip(pred, x_val_nop, real_y, pred_proba):
    if p == y:
        print(f'{labels_flip[y]} predicted as {labels_flip[p]}')
        print(f'probabilities: {labels_flip[0]} {np.round(probs[0], 3)}, \
                               \n {labels_flip[1]} {np.round(probs[1], 3)}, \
                               \n {labels_flip[2]} {np.round(probs[2], 3)}, \
                               \n {labels_flip[3]} {np.round(probs[3], 3)}, \
                               \n {labels_flip[4]} {np.round(probs[4], 3)}')
        plt.imshow(image)
        plt.show()

apartment predicted as apartment
probabilities: apartment 0.40799999237060547,                                
 house 0.1589999943971634,                                
 industrial 0.1120000034570694,                                
 retail 0.1599999964237213,                                
 officebuilding 0.16099999845027924

apartment predicted as apartment
probabilities: apartment 0.3959999978542328,                                
 house 0.164000004529953,                                
 industrial 0.10499999672174454,                                
 retail 0.16699999570846558,                                
 officebuilding 0.1679999977350235

apartment predicted as apartment
probabilities: apartment 0.5569999814033508,                                
 house 0.06300000101327896,                                
 industrial 0.06800000369548798,                                
 retail 0.12700000405311584,                                
 officebuilding 0.1860000044107437

apartment predicted as apartment
probabilities: apartment 0.7459999918937683,                                
 house 0.02500000037252903,                                
 industrial 0.017999999225139618,                                
 retail 0.06599999964237213,                                
 officebuilding 0.14399999380111694

apartment predicted as apartment
probabilities: apartment 0.4569999873638153,                                
 house 0.004000000189989805,                                
 industrial 0.032999999821186066,                                
 retail 0.0949999988079071,                                
 officebuilding 0.41100001335144043
In [192]:
#some examples of mislabeled images

for p, image, y, probs in zip(pred, x_val_nop, real_y, pred_proba):
    if p != y:
        print(f'{labels_flip[y]} predicted as {labels_flip[p]}')
        print(f'probabilities: {labels_flip[0]} {np.round(probs[0], 3)}, \
                               \n {labels_flip[1]} {np.round(probs[1], 3)}, \
                               \n {labels_flip[2]} {np.round(probs[2], 3)}, \
                               \n {labels_flip[3]} {np.round(probs[3], 3)}, \
                               \n {labels_flip[4]} {np.round(probs[4], 3)}')
        plt.imshow(image)
        plt.show()

apartment predicted as industrial
probabilities: apartment 0.09399999678134918,                                
 house 0.08299999684095383,                                
 industrial 0.5669999718666077,                                
 retail 0.18299999833106995,                                
 officebuilding 0.0729999989271164

apartment predicted as industrial
probabilities: apartment 0.11999999731779099,                                
 house 0.12399999797344208,                                
 industrial 0.36500000953674316,                                
 retail 0.2680000066757202,                                
 officebuilding 0.12399999797344208

apartment predicted as house
probabilities: apartment 0.17900000512599945,                                
 house 0.4659999907016754,                                
 industrial 0.1420000046491623,                                
 retail 0.125,                                
 officebuilding 0.08799999952316284

apartment predicted as industrial
probabilities: apartment 0.0,                                
 house 0.0,                                
 industrial 0.9990000128746033,                                
 retail 0.0,                                
 officebuilding 0.0

apartment predicted as house
probabilities: apartment 0.26499998569488525,                                
 house 0.6069999933242798,                                
 industrial 0.019999999552965164,                                
 retail 0.0560000017285347,                                
 officebuilding 0.052000001072883606

apartment predicted as house
probabilities: apartment 0.1809999942779541,                                
 house 0.39800000190734863,                                
 industrial 0.10100000351667404,                                
 retail 0.1979999989271164,                                
 officebuilding 0.12200000137090683

apartment predicted as house
probabilities: apartment 0.30399999022483826,                                
 house 0.4659999907016754,                                
 industrial 0.04600000008940697,                                
 retail 0.0949999988079071,                                
 officebuilding 0.09000000357627869

apartment predicted as officebuilding
probabilities: apartment 0.03799999877810478,                                
 house 0.0,                                
 industrial 0.0,                                
 retail 0.0,                                
 officebuilding 0.9620000123977661

apartment predicted as industrial
probabilities: apartment 0.15299999713897705,                                
 house 0.07599999755620956,                                
 industrial 0.37599998712539673,                                
 retail 0.24899999797344208,                                
 officebuilding 0.1459999978542328

apartment predicted as officebuilding
probabilities: apartment 0.0,                                
 house 0.0,                                
 industrial 0.0,                                
 retail 0.0,                                
 officebuilding 1.0

apartment predicted as officebuilding
probabilities: apartment 0.16899999976158142,                                
 house 0.004000000189989805,                                
 industrial 0.11400000005960464,                                
 retail 0.2619999945163727,                                
 officebuilding 0.45100000500679016

apartment predicted as industrial
probabilities: apartment 0.02199999988079071,                                
 house 0.014999999664723873,                                
 industrial 0.9340000152587891,                                
 retail 0.023000000044703484,                                
 officebuilding 0.006000000052154064

apartment predicted as retail
probabilities: apartment 0.14499999582767487,                                
 house 0.04899999871850014,                                
 industrial 0.28299999237060547,                                
 retail 0.32100000977516174,                                
 officebuilding 0.20200000703334808

apartment predicted as retail
probabilities: apartment 0.0020000000949949026,                                
 house 0.0,                                
 industrial 0.027000000700354576,                                
 retail 0.6819999814033508,                                
 officebuilding 0.289000004529953

apartment predicted as retail
probabilities: apartment 0.0,                                
 house 0.0,                                
 industrial 0.020999999716877937,                                
 retail 0.9330000281333923,                                
 officebuilding 0.04500000178813934

apartment predicted as retail
probabilities: apartment 0.03999999910593033,                                
 house 0.01899999938905239,                                
 industrial 0.3919999897480011,                                
 retail 0.4359999895095825,                                
 officebuilding 0.11299999803304672

apartment predicted as officebuilding
probabilities: apartment 0.16099999845027924,                                
 house 0.0,                                
 industrial 0.0,                                
 retail 0.004000000189989805,                                
 officebuilding 0.8360000252723694

apartment predicted as retail
probabilities: apartment 0.0,                                
 house 0.0,                                
 industrial 0.03400000184774399,                                
 retail 0.9079999923706055,                                
 officebuilding 0.05700000002980232

apartment predicted as retail
probabilities: apartment 0.006000000052154064,                                
 house 0.0020000000949949026,                                
 industrial 0.17599999904632568,                                
 retail 0.7160000205039978,                                
 officebuilding 0.10100000351667404

apartment predicted as retail
probabilities: apartment 0.003000000026077032,                                
 house 0.0,                                
 industrial 0.09200000017881393,                                
 retail 0.8090000152587891,                                
 officebuilding 0.09600000083446503

apartment predicted as industrial
probabilities: apartment 0.17800000309944153,                                
 house 0.08100000023841858,                                
 industrial 0.4099999964237213,                                
 retail 0.20100000500679016,                                
 officebuilding 0.12999999523162842

apartment predicted as house
probabilities: apartment 0.11400000005960464,                                
 house 0.8690000176429749,                                
 industrial 0.0010000000474974513,                                
 retail 0.008999999612569809,                                
 officebuilding 0.007000000216066837

apartment predicted as retail
probabilities: apartment 0.24799999594688416,                                
 house 0.17399999499320984,                                
 industrial 0.1340000033378601,                                
 retail 0.25200000405311584,                                
 officebuilding 0.19200000166893005

apartment predicted as industrial
probabilities: apartment 0.125,                                
 house 0.125,                                
 industrial 0.41999998688697815,                                
 retail 0.22300000488758087,                                
 officebuilding 0.10599999874830246

apartment predicted as officebuilding
probabilities: apartment 0.09099999815225601,                                
 house 0.0,                                
 industrial 0.0,                                
 retail 0.007000000216066837,                                
 officebuilding 0.9010000228881836

apartment predicted as industrial
probabilities: apartment 0.11800000071525574,                                
 house 0.2199999988079071,                                
 industrial 0.5350000262260437,                                
 retail 0.0820000022649765,                                
 officebuilding 0.04500000178813934

apartment predicted as house
probabilities: apartment 0.20800000429153442,                                
 house 0.46299999952316284,                                
 industrial 0.0860000029206276,                                
 retail 0.1420000046491623,                                
 officebuilding 0.10100000351667404
In [193]:
from sklearn.metrics import classification_report, confusion_matrix

print(classification_report(real_y, pred, target_names=labels))

                precision    recall  f1-score   support

     apartment       0.22      0.17      0.19       440
         house       0.22      0.22      0.22       440
    industrial       0.20      0.28      0.23       440
        retail       0.18      0.15      0.17       440
officebuilding       0.20      0.21      0.20       440

      accuracy                           0.21      2200
     macro avg       0.21      0.21      0.20      2200
  weighted avg       0.21      0.21      0.20      2200
In [194]:
sns.heatmap(confusion_matrix(real_y, pred))
plt.xlabel('predicted class')
plt.ylabel('real class')
plt.xticks(ticks=[x + 0.5 for x in list(labels_flip.keys())], labels=list(labels_flip.values()), rotation=90);
plt.yticks(ticks=[x + 0.5 for x in list(labels_flip.keys())], labels=list(labels_flip.values()), rotation=0);

Some notes on the results

  • industrial is the easiest one to classify correctly, followed by house.

  • retail and apartment are hardest to classify corretly (also by hand, I noticed, as they are the most ambiguous).

  • all classes are often classified mistakenly as industrial (it seems like the net has a tendency to classify a lot of things as industrial, hence maybe the high correctly classified industrial images).
  • On the contrary, very few images are classified as retail and apartments, the hardest to spot.\
In [ ]: